This is an R Markdown Notebook. When you execute code within the notebook, the results appear beneath the code.
Try executing this chunk by clicking the Run button within the chunk or by placing your cursor inside it and pressing Ctrl+Shift+Enter.
#BEGIN HERE—————-
library(data.table)
library(tidyverse)
library(lubridate)
file='/media/ad/01D6B57CFBE4DB20/1.Linux/Data/FQC/V04R-V04R-SQLData_3000_head.dat'
#file='/media/ad/01D6B57CFBE4DB20/1.Linux/Data/FQC/V04R-V04R-SQLData_3000.dat'# 3.6 GB
#file<-'/media/ad/01D6B57CFBE4DB20/1.Linux/Data/FQC/V04R-V04R-SQLData_E_series.dat' # 2.9 GB crack session (ram?)
#file<-'/media/ad/01D6B57CFBE4DB20/1.Linux/Data/FQC/V04R-V04R-SQLData_Ric.dat' # 2.8 GB OK
data_head<-fread(file,fill=TRUE,nrows=5)
data_head
#install.packages('doParallel')
#Method 1
#library(doParallel)
#no_cores <- detectCores()
#print(no_cores)
#cl <- makeCluster(no_cores)
#registerDoParallel(cl)
#Method 2
#library(doParallel)
#cl <- makePSOCKcluster(5)
#registerDoParallel(cl)
# method 3
setDTthreads(threads = 0)
getDTthreads(verbose=TRUE)
omp_get_num_procs() 2
R_DATATABLE_NUM_PROCS_PERCENT unset (default 50)
R_DATATABLE_NUM_THREADS unset
R_DATATABLE_THROTTLE unset (default 1024)
omp_get_thread_limit() 2147483647
omp_get_max_threads() 2
OMP_THREAD_LIMIT unset
OMP_NUM_THREADS unset
RestoreAfterFork true
data.table is using 2 threads with throttle==1024. See ?setDTthreads.
[1] 2
start.time <- Sys.time()
product_type_column='V3'
tryCatch(
expr = {dt1 <<- fread(file,fill=TRUE,select=c(product_type_column))},
warning = function(w){
cat('Warning: ', w$message, '\n\n');
n_line <- as.numeric(gsub('Stopped early on line (\\d+)\\..*','\\1',w$message))
if (!is.na(n_line)) {
cat('Found ', n_line,'\n')
dt1_part1 <- fread(file,fill=TRUE, nrows=n_line-1,select=c(product_type_column))
dt1_part2 <- fread(file,fill=TRUE,skip=n_line,select=c(product_type_column))
dt1 <<- rbind(dt1_part1, dt1_part2, fill=T)
}
},
finally = cat("\nFinished. \n")
#finally=.Call(CfreadCleanup)
);
Finished.
end.time <- Sys.time()
time.taken <- end.time - start.time
time.taken
Time difference of 0.2093291 secs
# No Parallel: 1.118 mins
# doParallel method 1: 1.097 mins --> run lan 2 la 58 or 52 s
# doParallel method 2: 1.118 mins
# doParallel method 3: 51 s
#install.packages("pryr")
library(pryr)
Registered S3 method overwritten by 'pryr':
method from
print.bytes Rcpp
Attaching package: ‘pryr’
The following objects are masked from ‘package:purrr’:
compose, partial
The following object is masked from ‘package:data.table’:
address
object_size(dt1) # 43.7 MB
405 kB
#product_type_column='V3'
#dt1<-fread(file,fill=TRUE,select=c(product_type_column))
dim(dt1)
[1] 50000 1
head(dt1)
tail(dt1)
class(dt1)
[1] "data.table" "data.frame"
product_all <- as.data.frame(table(dt1)) %>%
arrange(desc(Freq)) %>%
top_n(20,Freq)
product_all
product_all %>%
ggplot(aes(x= reorder(dt1, -Freq),y=Freq)) +
geom_bar(stat="identity",fill="steelblue")+
theme(axis.text.x=element_text(angle=90))
# Please choose product_type:
product_type='31AA015'
#product_type='31A007G_P'
# Please choose parameter column to show
parameter_column_name='V65'
parameter_column_value='V66'
# Default time column:
product_type_column='V3'
date_column='V6'
start.time <- Sys.time()
tryCatch(
expr = {dt2 <<- fread(file,fill=TRUE,select=c(product_type_column,date_column,parameter_column_name,parameter_column_value)) %>%
filter(V3 ==product_type)
},
warning = function(w){
cat('Warning: ', w$message, '\n\n');
n_line2 <- as.numeric(gsub('Stopped early on line (\\d+)\\..*','\\1',w$message))
if (!is.na(n_line2)) {
cat('Found ', n_line2,'\n')
dt2_part1 <- fread(file,fill=TRUE, nrows=n_line2-1,select=c(product_type_column,date_column,parameter_column_name,parameter_column_value))
dt2_part2 <- fread(file,fill=TRUE,skip=n_line2,select=c(product_type_column,date_column,
parameter_column_name,parameter_column_value))
dt2 <<- rbind(dt2_part1, dt2_part2, fill=T)%>%
filter(V3 ==product_type)
}
},
finally = cat("\nFinished. \n")
);
Finished.
end.time <- Sys.time()
time.taken <- end.time - start.time
time.taken # 1.1207 mins , run lan 2: 46.15 s
Time difference of 0.1885769 secs
# Method 3: 43 s
object_size(dt2) # 60 MB
754 kB
env <- environment()
all_data_memory <-data.frame(
object = ls(env),
size = unlist(lapply(
ls(env), function(x) {
object.size(get(x, envir = env, inherits = FALSE))
}
)))
all_data_memory
#60 Megabytes (MB) = 62,914,560 Bytes (B)
#Remove dt 1 to save ram
#rm(dt1)
dim(dt2)
[1] 6965 4
head(dt2)
tail(dt2)
colnames(dt2) <- c("product_type", "date","parameter_name","parameter_value")
dt2
print(dim(dt2))
[1] 6965 4
sl<-object.size(dt2)
print(sl, units = "auto", standard = "SI")
754 kB
data_process <- dt2 %>%
mutate( date_trans=mdy_hms(date), # must have mdy_hms for convert date time
date_filter=as_date(date_trans),
)
head(data_process)
tail(data_process)
print(dim(data_process))
[1] 6965 6
sl<-object.size(data_process)
print(sl, units = "auto", standard = "SI")
866.5 kB
#rm(dt2)
min(data_process$date)
[1] "12/18/2017 23:31:1"
max(data_process$date)
[1] "12/27/2017 23:30:1"
data_process[which.max(data_process$date)]
NAs introduced by coercion
class(data_process$date_trans)
[1] "POSIXct" "POSIXt"
class(data_process$date)
[1] "character"
min(data_process$date_trans)
[1] "2017-12-18 23:31:01 UTC"
max(data_process$date_trans)
[1] "2017-12-27 23:30:01 UTC"
dim(data_process)
[1] 6965 6
#data_process %>%
# filter(date_trans>=as.POSIXct("2017-12-20 23:30:01",tz="UTC"))
#please choose specific time to show
#year_select=2017
#month_select=12
#day_select=22
#df_plot <- data_process %>%
# select(date_trans,date,parameter_name,parameter_value,year,month,day) %>%
# filter(year==year_select,month==month_select,day==day_select)
#df_plot
# Best to choose
date_choose="2017-12-19"
df_plot2 <- data_process %>%
select(date_trans,date,date_filter,parameter_name,parameter_value) %>%
filter(date_filter==ymd(date_choose))
df_plot2
summary(df_plot2)
date_trans date date_filter parameter_name
Min. :2017-12-19 00:00:03 Length:1136 Min. :2017-12-19 Min. :500
1st Qu.:2017-12-19 00:38:14 Class :character 1st Qu.:2017-12-19 1st Qu.:500
Median :2017-12-19 01:15:41 Mode :character Median :2017-12-19 Median :500
Mean :2017-12-19 08:22:41 Mean :2017-12-19 Mean :500
3rd Qu.:2017-12-19 22:50:21 3rd Qu.:2017-12-19 3rd Qu.:500
Max. :2017-12-19 23:35:56 Max. :2017-12-19 Max. :500
parameter_value
Min. :104.0
1st Qu.:106.9
Median :107.0
Mean :107.0
3rd Qu.:107.1
Max. :107.5
#install.packages('pastecs')
library(pastecs)
Attaching package: ‘pastecs’
The following objects are masked from ‘package:xts’:
first, last
The following objects are masked from ‘package:dplyr’:
first, last
The following object is masked from ‘package:tidyr’:
extract
The following objects are masked from ‘package:data.table’:
first, last
stat.desc(df_plot2 %>% select(parameter_name,parameter_value))
#install.packages('Hmisc')
library(Hmisc)
Loading required package: lattice
Loading required package: survival
Loading required package: Formula
Attaching package: ‘Hmisc’
The following object is masked from ‘package:psych’:
describe
The following object is masked from ‘package:plotly’:
subplot
The following objects are masked from ‘package:dplyr’:
src, summarize
The following objects are masked from ‘package:base’:
format.pval, units
describe(df_plot2 %>% select(parameter_name,parameter_value))
df_plot2 %>% select(parameter_name, parameter_value)
2 Variables 1136 Observations
----------------------------------------------------------------------------------------
parameter_name
n missing distinct Info Mean Gmd
1136 0 1 0 500 0
Value 500
Frequency 1136
Proportion 1
----------------------------------------------------------------------------------------
parameter_value
n missing distinct Info Mean Gmd .05 .10 .25
1136 0 588 1 107 0.2346 106.6 106.7 106.9
.50 .75 .90 .95
107.0 107.1 107.2 107.3
lowest : 104.018 104.333 105.644 105.825 106.103, highest: 107.438 107.442 107.453 107.466 107.511
----------------------------------------------------------------------------------------
#install.packages('psych')
library(psych)
describe(df_plot2 %>% select(parameter_name,parameter_value))
df_plot2 %>% select(parameter_name, parameter_value)
2 Variables 1136 Observations
----------------------------------------------------------------------------------------
parameter_name
n missing distinct Info Mean Gmd
1136 0 1 0 500 0
Value 500
Frequency 1136
Proportion 1
----------------------------------------------------------------------------------------
parameter_value
n missing distinct Info Mean Gmd .05 .10 .25
1136 0 588 1 107 0.2346 106.6 106.7 106.9
.50 .75 .90 .95
107.0 107.1 107.2 107.3
lowest : 104.018 104.333 105.644 105.825 106.103, highest: 107.438 107.442 107.453 107.466 107.511
----------------------------------------------------------------------------------------
df_plot2 %>% select(parameter_value) %>%
ggplot(aes(y=parameter_value))+
geom_histogram()
# library(plotly)
# ay <- list(
# tickfont = list(color = "red"),
# overlaying = "y",
# side = "right",
# title = "second y axis"
# )
# p <- df_plot2 %>%
# #arrange(date_trans) %>%
# plot_ly(x =~date_trans, y = ~parameter_value,name='value',
# type='scatter',mode='lines+markers')%>%
# add_trace(x=~date_trans, y = ~parameter_name,name='name',
# type='scatter',mode='lines',yaxis='y2') %>%
# layout(yaxis2 = list(overlaying = "y", side = "right")
# )
# p
#p1<-plot_ly(df_plot2,x =~date_trans, y = ~parameter_value) %>%
# add_lines(name='Value') %>%
#p2<-plot_ly(df_plot2,x =~date_trans, y = ~parameter_name) %>%
# add_lines(name='Frequency')
#subplot(p1, p2,nrows=2,shareX = TRUE)
library(plotly)
p1<-plot_ly(df_plot2,x =~date_trans, y = ~parameter_value) %>%
add_trace(mode='lines+markers',name='Value')
p2<-plot_ly(df_plot2,x =~date_trans, y = ~parameter_name) %>%
add_trace(mode='lines+markers',name='Frequency')
plotly::subplot(p1, p2,nrows=2,shareX = TRUE)
No trace type specified:
Based on info supplied, a 'scatter' trace seems appropriate.
Read more about this trace type -> https://plot.ly/r/reference/#scatter
No trace type specified:
Based on info supplied, a 'scatter' trace seems appropriate.
Read more about this trace type -> https://plot.ly/r/reference/#scatter
select_data<-df_plot2$parameter_value
Q1 <- quantile(select_data, .25)
Q3 <- quantile(select_data, .75)
IQR <- IQR(select_data)
k=3
df_plot_no_outlier <- subset(df_plot2, select_data> (Q1 - k*IQR) & select_data< (Q3 + k*IQR))
print(dim(df_plot2))
[1] 1136 5
print(dim(df_plot_no_outlier))
[1] 1132 5
p1<-plot_ly(df_plot_no_outlier,x =~date_trans, y = ~parameter_value) %>%
add_trace(mode='lines+markers',name='Value')
p2<-plot_ly(df_plot_no_outlier,x =~date_trans, y = ~parameter_name) %>%
add_trace(mode='lines+markers',name='Frequency')
p<-plotly::subplot(p1, p2,nrows=2,shareX = TRUE)
No trace type specified:
Based on info supplied, a 'scatter' trace seems appropriate.
Read more about this trace type -> https://plot.ly/r/reference/#scatter
No trace type specified:
Based on info supplied, a 'scatter' trace seems appropriate.
Read more about this trace type -> https://plot.ly/r/reference/#scatter
p
p1<-plot_ly(df_plot_no_outlier,x =~date_trans, y = ~parameter_value) %>%
add_trace(mode='lines+markers',name='Value')
#p2<-plot_ly(df_plot_no_outlier,x =~date_trans, y = ~parameter_name) %>%
# add_trace(mode='lines+markers',name='Frequency')
#p<-subplot(p1, p2,nrows=2,shareX = TRUE)
p1
No trace type specified:
Based on info supplied, a 'scatter' trace seems appropriate.
Read more about this trace type -> https://plot.ly/r/reference/#scatter
No trace type specified:
Based on info supplied, a 'scatter' trace seems appropriate.
Read more about this trace type -> https://plot.ly/r/reference/#scatter
p1<-plot_ly(df_plot_no_outlier,x =~date_trans, y = ~parameter_value) %>%
add_trace(mode='lines+markers',name='Value') %>%
layout(
title = "FQC data daily",
xaxis = list(
rangeselector = list(
buttons = list(
list(
count = 30,
label = "min",
step = "minute",
stepmode = "backward"),
list(
count = 2,
label = "hour",
step = "hour",
stepmode = "backward"),
list(step = "all"))),
rangeslider = list(type = "date")),
yaxis = list(title = "Value"))
p1
No trace type specified:
Based on info supplied, a 'scatter' trace seems appropriate.
Read more about this trace type -> https://plot.ly/r/reference/#scatter
No trace type specified:
Based on info supplied, a 'scatter' trace seems appropriate.
Read more about this trace type -> https://plot.ly/r/reference/#scatter
# ay <- list(
# tickfont = list(color = "red"),
# overlaying = "y",
# side = "right",
# title = "second y axis"
# )
# p <- df_plot_no_outlier %>%
# arrange(date_trans) %>%
# plot_ly(x =~date_trans, y = ~parameter_value,name='value',
# type='scatter',mode='lines+markers')%>%
# add_trace(x=~date_trans, y = ~parameter_name,name='name',
# type='scatter',mode='lines',yaxis='y2') %>%
# layout(yaxis2 = list(overlaying = "y", side = "right")
# )
# p
num_points=dim(df_plot_no_outlier)[1]
num_points
[1] 1132
USL_value=''
LSL_value=''
LSL<-rep(LSL_value,num_points)
USL<-rep(USL_value,num_points)
p1%>%
add_trace(y = ~USL, name = 'USL', mode = 'lines',color="rgba(255, 0, 0, 0.6)") %>%
add_trace(y = ~LSL, name = 'LSL', mode = 'lines',color="rgb(20, 20, 20)")
No trace type specified:
Based on info supplied, a 'scatter' trace seems appropriate.
Read more about this trace type -> https://plot.ly/r/reference/#scatter
No trace type specified:
Based on info supplied, a 'bar' trace seems appropriate.
Read more about this trace type -> https://plot.ly/r/reference/#bar
No trace type specified:
Based on info supplied, a 'bar' trace seems appropriate.
Read more about this trace type -> https://plot.ly/r/reference/#bar
minimal value for n is 3, returning requested palette with 3 different levels
minimal value for n is 3, returning requested palette with 3 different levels
Can't display both discrete & non-discrete data on same axis'bar' objects don't have these attributes: 'mode'
Valid attributes include:
'type', 'visible', 'showlegend', 'legendgroup', 'opacity', 'name', 'uid', 'ids', 'customdata', 'meta', 'selectedpoints', 'hoverinfo', 'hoverlabel', 'stream', 'transforms', 'uirevision', 'x', 'x0', 'dx', 'y', 'y0', 'dy', 'text', 'texttemplate', 'hovertext', 'hovertemplate', 'textposition', 'insidetextanchor', 'textangle', 'textfont', 'insidetextfont', 'outsidetextfont', 'constraintext', 'cliponaxis', 'orientation', 'base', 'offset', 'width', 'marker', 'offsetgroup', 'alignmentgroup', 'selected', 'unselected', 'r', 't', '_deprecated', 'error_x', 'error_y', 'xcalendar', 'ycalendar', 'xaxis', 'yaxis', 'idssrc', 'customdatasrc', 'metasrc', 'hoverinfosrc', 'xsrc', 'ysrc', 'textsrc', 'texttemplatesrc', 'hovertextsrc', 'hovertemplatesrc', 'textpositionsrc', 'basesrc', 'offsetsrc', 'widthsrc', 'rsrc', 'tsrc', 'key', 'set', 'frame', 'transforms', '_isNestedKey', '_isSimpleKey', '_isGraticule', '_bbox'
'bar' objects don't have these attributes: 'mode'
Valid attributes include:
'type', 'visible', 'showlegend', 'legendgroup', 'opacity', 'name', 'uid', 'ids', 'customdata', 'meta', 'selectedpoints', 'hoverinfo', 'hoverlabel', 'stream', 'transforms', 'uirevision', 'x', 'x0', 'dx', 'y', 'y0', 'dy', 'text', 'texttemplate', 'hovertext', 'hovertemplate', 'textposition', 'insidetextanchor', 'textangle', 'textfont', 'insidetextfont', 'outsidetextfont', 'constraintext', 'cliponaxis', 'orientation', 'base', 'offset', 'width', 'marker', 'offsetgroup', 'alignmentgroup', 'selected', 'unselected', 'r', 't', '_deprecated', 'error_x', 'error_y', 'xcalendar', 'ycalendar', 'xaxis', 'yaxis', 'idssrc', 'customdatasrc', 'metasrc', 'hoverinfosrc', 'xsrc', 'ysrc', 'textsrc', 'texttemplatesrc', 'hovertextsrc', 'hovertemplatesrc', 'textpositionsrc', 'basesrc', 'offsetsrc', 'widthsrc', 'rsrc', 'tsrc', 'key', 'set', 'frame', 'transforms', '_isNestedKey', '_isSimpleKey', '_isGraticule', '_bbox'
No trace type specified:
Based on info supplied, a 'scatter' trace seems appropriate.
Read more about this trace type -> https://plot.ly/r/reference/#scatter
No trace type specified:
Based on info supplied, a 'bar' trace seems appropriate.
Read more about this trace type -> https://plot.ly/r/reference/#bar
No trace type specified:
Based on info supplied, a 'bar' trace seems appropriate.
Read more about this trace type -> https://plot.ly/r/reference/#bar
minimal value for n is 3, returning requested palette with 3 different levels
minimal value for n is 3, returning requested palette with 3 different levels
Can't display both discrete & non-discrete data on same axis'bar' objects don't have these attributes: 'mode'
Valid attributes include:
'type', 'visible', 'showlegend', 'legendgroup', 'opacity', 'name', 'uid', 'ids', 'customdata', 'meta', 'selectedpoints', 'hoverinfo', 'hoverlabel', 'stream', 'transforms', 'uirevision', 'x', 'x0', 'dx', 'y', 'y0', 'dy', 'text', 'texttemplate', 'hovertext', 'hovertemplate', 'textposition', 'insidetextanchor', 'textangle', 'textfont', 'insidetextfont', 'outsidetextfont', 'constraintext', 'cliponaxis', 'orientation', 'base', 'offset', 'width', 'marker', 'offsetgroup', 'alignmentgroup', 'selected', 'unselected', 'r', 't', '_deprecated', 'error_x', 'error_y', 'xcalendar', 'ycalendar', 'xaxis', 'yaxis', 'idssrc', 'customdatasrc', 'metasrc', 'hoverinfosrc', 'xsrc', 'ysrc', 'textsrc', 'texttemplatesrc', 'hovertextsrc', 'hovertemplatesrc', 'textpositionsrc', 'basesrc', 'offsetsrc', 'widthsrc', 'rsrc', 'tsrc', 'key', 'set', 'frame', 'transforms', '_isNestedKey', '_isSimpleKey', '_isGraticule', '_bbox'
'bar' objects don't have these attributes: 'mode'
Valid attributes include:
'type', 'visible', 'showlegend', 'legendgroup', 'opacity', 'name', 'uid', 'ids', 'customdata', 'meta', 'selectedpoints', 'hoverinfo', 'hoverlabel', 'stream', 'transforms', 'uirevision', 'x', 'x0', 'dx', 'y', 'y0', 'dy', 'text', 'texttemplate', 'hovertext', 'hovertemplate', 'textposition', 'insidetextanchor', 'textangle', 'textfont', 'insidetextfont', 'outsidetextfont', 'constraintext', 'cliponaxis', 'orientation', 'base', 'offset', 'width', 'marker', 'offsetgroup', 'alignmentgroup', 'selected', 'unselected', 'r', 't', '_deprecated', 'error_x', 'error_y', 'xcalendar', 'ycalendar', 'xaxis', 'yaxis', 'idssrc', 'customdatasrc', 'metasrc', 'hoverinfosrc', 'xsrc', 'ysrc', 'textsrc', 'texttemplatesrc', 'hovertextsrc', 'hovertemplatesrc', 'textpositionsrc', 'basesrc', 'offsetsrc', 'widthsrc', 'rsrc', 'tsrc', 'key', 'set', 'frame', 'transforms', '_isNestedKey', '_isSimpleKey', '_isGraticule', '_bbox'
Try another chart: High chart
head(df_plot_no_outlier)
TRY OTHER METHOD:
#install.packages('highcharter')
library(highcharter)
Registered S3 method overwritten by 'quantmod':
method from
as.zoo.data.frame zoo
dt2%>%
hchart(type = "line", hcaes(x = V6, y = V66))
Error: Problem with `mutate()` input `x`.
x object 'V6' not found
ℹ Input `x` is `V6`.
Run `rlang::last_error()` to see where the error occurred.
df_plot_no_outlier%>%
hchart(type = "line", hcaes(x = as.character(date_trans), y = parameter_value))
#install.packages("dygraphs")
library(dygraphs)
df_plot_no_outlier
dt2
Sys.timezone()
[1] "Asia/Ho_Chi_Minh"
tzone(data_time$date_trans)
[1] "UTC"
tzone(df_plot_no_outlier$date_trans)
[1] "UTC"
# Switch to XTS format
data <- xts(x = dt2$V66,
order.by = mdy_hms(dt2$V6))
# Add points
p <- dygraph(data) %>%
dyOptions( drawPoints = TRUE, pointSize = 4,useDataTimezone = TRUE )%>%
dyRangeSelector()
p
# Switch to XTS format
data <- xts(x = df_plot_no_outlier$parameter_value,
order.by = df_plot_no_outlier$date_trans)
# Add points
p <- dygraph(data) %>%
dyOptions( drawPoints = TRUE, pointSize = 4,useDataTimezone = TRUE )%>%
dyRangeSelector()
p
df_plot_no_outlier
dygraph(df_plot_no_outlier %>% select(date_trans,parameter_value), main = "All", group = "lung-deaths")%>%
dyOptions( drawPoints = TRUE, pointSize = 4,useDataTimezone = TRUE )
dygraph(df_plot_no_outlier %>% select(date_trans,parameter_name), main = "All", group = "lung-deaths")%>%
dyOptions( drawPoints = TRUE, pointSize = 4,useDataTimezone = TRUE )
dy_graph <- list(
dygraph(df_plot_no_outlier %>% select(date_trans,parameter_value), main = "temperature", group = "lung-deaths")%>%
dyOptions( drawPoints = TRUE, pointSize = 4,useDataTimezone = TRUE ),
dygraph(df_plot_no_outlier %>% select(date_trans,parameter_name), main = "rainfall", group = "lung-deaths")%>%
dyOptions( drawPoints = TRUE, pointSize = 4,useDataTimezone = TRUE )
) # end list
# render the dygraphs objects using htmltools
htmltools::browsable(htmltools::tagList(dy_graph))
# Please choose product_type:
file='/media/ad/01D6B57CFBE4DB20/1.Linux/Data/FQC/V04R-V04R-SQLData_3000_head.dat'
#product_type='31AA015'
product_type_choose='31AA015'
#product_type='31A007G_P'
# Please choose parameter column to show
parameter_column_name='V65'
parameter_column_value='V66'
# Default time column:
product_type_column='V3'
date_column='V6'
savefunc2 <- function(file_path,product_type_column,product_type_choose=NULL,date_column=NULL,
parameter_column_name=NULL,parameter_column_value=NULL){
tryCatch(
expr = {
if ( is.null(product_type_choose)){
print('not filter')
dt2 <<- fread(file_path,fill=TRUE,select=c(product_type_column,date_column,parameter_column_name,parameter_column_value))}
else{
print('filter')
dt2 <<- fread(file_path,fill=TRUE,select=c(product_type_column,date_column,parameter_column_name,parameter_column_value)) %>%
filter(V3 ==product_type_choose)}
},
warning = function(w){
cat('Warning: ', w$message, '\n\n');
n_line2 <- as.numeric(gsub('Stopped early on line (\\d+)\\..*','\\1',w$message))
if (!is.na(n_line2)) {
cat('Found ', n_line2,'\n')
dt2_part1 <- fread(file_path,fill=TRUE, nrows=n_line2-1,select=c(product_type_column,date_column,parameter_column_name,parameter_column_value))
dt2_part2 <- fread(file_path,fill=TRUE,skip=n_line2,select=c(product_type_column,date_column,parameter_column_name,parameter_column_value))
if ( is.null(product_type_choose)){
dt2 <<- rbind(dt2_part1, dt2_part2, fill=T)
}else{
dt2 <<- rbind(dt2_part1, dt2_part2, fill=T)%>%
filter(V3 ==product_type_choose)
}
}
},
finally = cat("\nFinished. \n")
);
}
dt<-savefunc2(file,product_type_column,product_type_choose,date_column,parameter_column_name,parameter_column_value)
colnames(dt) <- c("product_type", "date","parameter_freq","parameter_value")
dt<-dt %>%
mutate( date_trans=mdy_hms(date), # must have mdy_hms for convert date time
date_filter=as_date(date_trans),
)
dt
dt<-savefunc2(file,product_type_column)
dt
plotly_chart <- function(data_one_date,check_input_remove_frequency_chart,
check_go_data_analyze_date){
req(check_go_data_analyze_date)
p1<-plot_ly(data_one_date,x =~date_trans, y = ~parameter_value) %>%
add_trace(mode='lines+markers',name='Value')
if (check_input_remove_frequency_chart){
p1
} else {
p2<-plot_ly(data_one_date,x =~date_trans, y = ~parameter_freq) %>%
add_trace(mode='lines+markers',name='Frequency')
plotly::subplot(p1, p2,nrows=2,shareX = TRUE)
}
}
plotly_chart(data_one_date(),input$remove_frequency_chart,input$go_data_analyze_date)